{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "\n",
    "from selenium import webdriver\n",
    "import time\n",
    "import pymysql\n",
    "\n",
    "# 需要开多线程么？\n",
    "from selenium.webdriver.remote.webelement import WebElement\n",
    "\n",
    "chrome = webdriver.Chrome()\n",
    "conn = pymysql.connect(host='gsy-cloud', user='root', password='rootQWE', database='python', port=32769)\n",
    "cursor = conn.cursor()\n",
    "\n",
    "select_sql = \"select pro_pk_id,rate_num,is_visit from tb_teacher where is_visit = '0' or is_visit = '1';\"\n",
    "update_sql = \"update tb_teacher set isvisit = '2' where pre_pk_id = %d ;\"\n",
    "insert_sql = \"insert into tb_rate_comment\"\n",
    "\n",
    "\n",
    "def get_page(pk_id):\n",
    "    url = 'https://www.ratemyprofessors.com/ShowRatings.jsp?tid=replacepkid&showMyProfs=true'\n",
    "    chrome.get(url.replace('replacepkid', str(pk_id)))\n",
    "    try:\n",
    "        chrome.find_element_by_xpath('/html/body/div[5]/div/div/button').click()\n",
    "    except:\n",
    "        pass\n",
    "    load_more()\n",
    "\n",
    "\n",
    "def load_more():\n",
    "    while True:\n",
    "        try:\n",
    "            chrome.execute_script('arguments[0].click();', chrome.find_element_by_xpath(\n",
    "                \"//button[@class='Buttons__Button-sc-19xdot-1 PaginationButton__StyledPaginationButton-txi1dr-1 eaZArN']\"))\n",
    "            time.sleep(10)\n",
    "        except Exception as e:\n",
    "            if not str(e).find('no such element'):\n",
    "                print(e)\n",
    "            break\n",
    "\n",
    "\n",
    "def parse_page(id):\n",
    "    divs_list = chrome.find_elements_by_xpath(\n",
    "        \"//*[@id=\\\"ratingsList\\\"]/li/div/div[@class='Rating__RatingBody-sc-1rhvpxz-0 dGrvXb']\")\n",
    "    count = 1\n",
    "    for div in divs_list:\n",
    "        quality = parse_item(div, '/div[2]/div[1]/div[2]')\n",
    "        difficulty = parse_item(div, '/div[2]/div[2]/div[2]')\n",
    "        thumbs = parse_items(div, \"//div[@class='RatingFooter__HelpTotal-ciwspm-2 kAVFzA']\")\n",
    "        thumbs_up_num, thumbs_down_num = thumbs_ignore_out_of_index(thumbs)\n",
    "        course_id = parse_item(div, \"//div[@class='RatingHeader__StyledClass-sc-1dlkqw1-2 gxDIt']\")\n",
    "        comment = parse_item(div, \"//div[@class='Comments__StyledComments-dzzyvm-0 gRjWel']\")\n",
    "        comment_phrase = parse_phrase(parse_item(div, \"//div[@class='RatingTags__StyledTags-sc-1boeqx2-0 eLpnFv']\"))\n",
    "        date = parse_item(div,\n",
    "                          \"]//div[@class='TimeStamp__StyledTimeStamp-sc-9q2r30-0 bXQmMr RatingHeader__RatingTimeStamp-sc-1dlkqw1-3 BlaCV']\")\n",
    "        tuple_obj = (\n",
    "            str(id), str(count), quality, difficulty, thumbs_up_num, thumbs_down_num, course_id, comment,\n",
    "            comment_phrase,\n",
    "            date)\n",
    "        count += 1\n",
    "        print(tuple_obj)\n",
    "\n",
    "\n",
    "def thumbs_ignore_out_of_index(items: List[WebElement]):\n",
    "    try:\n",
    "        return items[0].text, items[1].text\n",
    "    except:\n",
    "        print(e)\n",
    "        return 0, 0\n",
    "\n",
    "\n",
    "def parse_phrase(item: WebElement):\n",
    "    phrase_list = parse_items(item, \"/span\")\n",
    "    ll = []\n",
    "    for i in phrase_list:\n",
    "        ll.append(i.text)\n",
    "    return '$$'.join(ll)\n",
    "\n",
    "\n",
    "def parse_item(div: WebElement, xpath_str):\n",
    "    try:\n",
    "        item = div.find_element_by_xpath(xpath_str)\n",
    "        return item.text\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return ''\n",
    "\n",
    "\n",
    "def parse_items(div: WebElement, xpath_str):\n",
    "    try:\n",
    "        item = div.find_elements_by_xpath(xpath_str)\n",
    "        return item\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return []\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_page(286226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CS512\n",
      "CS577\n",
      "CS512\n",
      "CS512\n",
      "CS512\n",
      "CS512\n",
      "CS411\n",
      "CS584\n",
      "CS584\n",
      "CS512CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS584\n",
      "CS411\n",
      "CS584\n",
      "CS584\n",
      "CS512\n",
      "CS587\n",
      "CS411\n",
      "CS411\n",
      "CS100\n",
      "CS411\n",
      "CS411\n",
      "CS100\n"
     ]
    }
   ],
   "source": [
    "id = 123\n",
    "divs_list = chrome.find_elements_by_xpath(\n",
    "    \"//*[@id=\\\"ratingsList\\\"]/li/div/div[@class='Rating__RatingBody-sc-1rhvpxz-0 dGrvXb']\")\n",
    "count = 1\n",
    "for div in divs_list:\n",
    "#     print(1)\n",
    "#     quality = parse_item(div, './div[2]/div[1]/div[2]')\n",
    "#     print(quality)\n",
    "#     print(quality)\n",
    "#     difficulty = parse_item(div, './div[2]/div[2]/div[2]')\n",
    "#     print(difficulty)\n",
    "\n",
    "#     thumbs = parse_items(div, \".//div[@class='RatingFooter__HelpTotal-ciwspm-2 kAVFzA']\")\n",
    "#     thumbs_up_num, thumbs_down_num = thumbs_ignore_out_of_index(thumbs)\n",
    "#     print(thumbs_up_num)\n",
    "\n",
    "\n",
    "#     course_id = parse_item(div, \".//div[@class='RatingHeader__StyledClass-sc-1dlkqw1-2 gxDIt']\")\n",
    "    course_id = div.find_element_by_xpath(\"./div[3]/div[1]/div[1]/div[1]\")\n",
    "    print(course_id.text)\n",
    "#     course_id = div.find_element_by_xpath(\".//div[@class='RatingHeader__StyledClass-sc-1dlkqw1-2 gxDIt']/text()\")\n",
    "#     print(course_id)\n",
    "#     comment = parse_item(div, \".//div[@class='Comments__StyledComments-dzzyvm-0 gRjWel']\")\n",
    "#     comment_phrase = parse_phrase(div, \".//div[@class='RatingTags__StyledTags-sc-1boeqx2-0 eLpnFv']\")\n",
    "#     print(comment_phrase)\n",
    "#     date = parse_item(div,\n",
    "#                       \".//div[@class='TimeStamp__StyledTimeStamp-sc-9q2r30-0 bXQmMr RatingHeader__RatingTimeStamp-sc-1dlkqw1-3 BlaCV']\")\n",
    "#     tuple_obj = (\n",
    "#         str(id), str(count), quality, difficulty, thumbs_up_num, thumbs_down_num, course_id, comment,\n",
    "#         comment_phrase,\n",
    "#         date)\n",
    "    count += 1\n",
    "#     print(tuple_obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_phrase(div: WebElement,xpath):\n",
    "    try:\n",
    "        item = div.find_element_by_xpath(xpath)\n",
    "#         print(type(item.text))\n",
    "        ss = item.text.split('\\n')\n",
    "#         phrase_list = parse_items(item, \"/span\")\n",
    "        ll = []\n",
    "        for i in ss:\n",
    "            ll.append(i)\n",
    "        return '$$'.join(ll)\n",
    "    except:\n",
    "        return ''"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
